# coding=utf-8

import threading
from datetime import datetime
from time import sleep

from difflib import SequenceMatcher


class Info:

    def __init__(self, key, count):
        self.key = key
        self.count = count
        
        self.list = [key]

    def __str__(self):
        ret = "%s: %s" % (self.count, self.key)
        for str1 in self.list:
            ret += ("\n    %s" % str1)
        ret += "\n"
        return ret 


def p(str1):
    if 0:
        print("%s" % str1)


def isSimilar(str1, str2):
    return SequenceMatcher(None, str1, str2).quick_ratio() >= 0.8

    
def _wordcount(sub_list, str1):

    for info in sub_list:
        if isSimilar(str1, info.key):
            info.count += 1
            info.list.append(str1)
            return
        
    sub_list.append(Info(str1, 1))


flag_merge = 1  # 目标是thread_count


def do_str_match(all_list, index, lines, start, end, ready_list):
    
    p("info: do_str_match(%s, %s) BEGIN" % (start, end))
    
    for i in range(start, end + 1):
        _wordcount(all_list[index], lines[i])

    p("info: do_str_match(%s, %s) DONE" % (start, end))
    
    ready_list.append(index)


def do_merge(all_list, index1, index2, ready_list):
    
    global flag_merge
    
    sub_list1 = all_list[index1]
    sub_list2 = all_list[index2]
    
    for info1 in sub_list1:
        for i in range(len(sub_list2)):
#         for info2 in sub_list2:
            if isSimilar(info1.key, sub_list2[i].key):
                info1.count += sub_list2[i].count
                info1.list.extend(sub_list2[i].list)
                sub_list2.__delitem__(i)
                break
    
    sub_list1.extend(sub_list2)
    
    ready_list.append(index1)
    flag_merge += 1
    
    
if __name__ == '__main__':

    # 1.读文件
    with open('7000.txt', 'rb') as f:
        lines = f.readlines()
    
    # 2.分段
    nrows = len(lines)
    count = 20
    part = nrows // count
    
    all_list = [[] for i in range(count)]
    ready_list = []
    
    # 3.分段统计
    threads = []
    for i in range(0, count):
        begin = i * part
        if i != count - 1:
            end = (i + 1) * part - 1
        else:
            end = nrows - 1
        
        t = threading.Thread(target=do_str_match, args=(all_list, i, lines, begin, end, ready_list))
        threads.append(t)

    print(datetime.now())

    for t in threads:
        t.start()
        
    # 4.周期轮巡
    last_index = 999
    while True:
        sleep(5)
        
        # 退出条件：分段统计完成 & 字典合并完成
        print("info: flag_merge=%s len(ready_list)=%s" % (flag_merge, len(ready_list)))
        if flag_merge == count:
            print("break!")
            break
        
        # 字典合并
        while len(ready_list) > 1:
            index0 = ready_list[0]
            index1 = ready_list[1]
            if len(all_list[index0]) >= len(all_list[index1]):
                t = threading.Thread(target=do_merge, args=(all_list, index0, index1, ready_list))
                last_index = index0
            else:
                t = threading.Thread(target=do_merge, args=(all_list, index1, index0, ready_list))
                last_index = index1
            threads.append(t)
            t.start()
            
            ready_list.__delitem__(0)
            ready_list.__delitem__(0)
#             ready_list = ready_list[2:]
          
    for t in threads:
        t.join()

    print(datetime.now())
        
    print("last_index=%s" % last_index)
    # 5.排序
    result = sorted(all_list[last_index], key=lambda x:x.count, reverse=True)
    
    print(datetime.now())
    
    # 6.校验总和
    sum1 = 0;
    for info in result:
        print(info)
        sum1 += info.count
    print("info: sum=%s" % sum1)
        
    print("*** OVER ***")

